import pandas as pd
import numpy as np


trn = train = pd.read_hdf('./input/test_d.h5')
feas = ['date', 'hour', 'weekday', 'month', 'xd',
       'yd', 'vd', 'dd', 'sd', 'xd_x_yd', 'xd_yd', 'yd/xd',
       'xd/yd', 'xd/s', 'yd/s', 'vd/s', 'dd/s', 'xd_x_yd/s', 'xd_yd/s',
       'v_x_s', 'd_x_s', 'xd_x_s', 'yd_x_s']


def cntv(v):
    def inner(x):
        xarr = x.tolist()
        return xarr.count(v)
    return inner


def per(q):
    def inner(x):
        val = x.quantile(q)
        return val
    return inner


per_num = 100
per_name = ['{:.1%}'.format(q) for q in np.linspace(0, 1, per_num)]
per_ops = [per(q) for q in np.linspace(0, 1, per_num)]
agg_names = ['count', 'sum', 'mean', 'std', 'skew', 'cnt0'] + per_name
agg_ops = ['count', 'sum', 'mean', 'std', 'skew', cntv(0)] + per_ops
trn1 = trn.drop_duplicates('ship')
for fea in feas:
    M = {}
    if fea in ['date', 'weekday', 'month', 'hour']:
        continue
    for name, op in zip(agg_names, agg_ops):
        M[fea+'_'+name] = op
    tmp = trn.groupby('ship')[fea].agg(M).reset_index()
    trn1 = pd.merge(trn1, tmp, on='ship', how='left', suffixes=('', ''))

features = [a for a in trn1.columns if a not in ['ship']] + ['ship']
trn2 = trn1[features]
trn2.replace({np.inf: 0, -np.inf: 0, np.nan: 0}, inplace=True)


trn2_0 = trn2.apply(cntv(0))
trn2_0_cols = trn2_0[trn2_0 < 7000].index
print(len(trn2_0_cols))
trn2[trn2_0_cols].to_csv(f'data/X_xyvd_tst_d2_{per_num}.csv', header=True, index=False)

